{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers, optimizers, datasets\n",
    "\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # or any {'0', '1', '2'}\n",
    "\n",
    "def mnist_dataset():\n",
    "    (x, y), (x_test, y_test) = datasets.mnist.load_data()\n",
    "    #normalize\n",
    "    x = x/255.0\n",
    "    x_test = x_test/255.0\n",
    "    \n",
    "    return (x, y), (x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Demo numpy based auto differentiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class Matmul:\n",
    "    def __init__(self):\n",
    "        self.mem = {}\n",
    "        \n",
    "    def forward(self, x, W):\n",
    "        h = np.matmul(x, W)\n",
    "        self.mem={'x': x, 'W':W}\n",
    "        return h\n",
    "    \n",
    "    def backward(self, grad_y):\n",
    "        '''\n",
    "        x: shape(N, d)\n",
    "        w: shape(d, d')\n",
    "        grad_y: shape(N, d')\n",
    "        '''\n",
    "        x = self.mem['x']\n",
    "        W = self.mem['W']\n",
    "        \n",
    "        ####################\n",
    "        grad_x = np.matmul(grad_y, W.T)\n",
    "        grad_W = np.matmul(x.T, grad_y)\n",
    "        ####################\n",
    "        return grad_x, grad_W\n",
    "\n",
    "\n",
    "class Relu:\n",
    "    def __init__(self):\n",
    "        self.mem = {}\n",
    "        \n",
    "    def forward(self, x):\n",
    "        self.mem['x']=x\n",
    "        return np.where(x > 0, x, np.zeros_like(x))\n",
    "    \n",
    "    def backward(self, grad_y):\n",
    "        '''\n",
    "        grad_y: same shape as x\n",
    "        '''\n",
    "        ####################\n",
    "        x = self.mem['x']\n",
    "        grad_x = grad_y * (x > 0)\n",
    "        ####################\n",
    "        return grad_x\n",
    "    \n",
    "\n",
    "\n",
    "class Softmax:\n",
    "    '''\n",
    "    softmax over last dimention\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        self.epsilon = 1e-12\n",
    "        self.mem = {}\n",
    "        \n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        x: shape(N, c)\n",
    "        '''\n",
    "        x_exp = np.exp(x)\n",
    "        partition = np.sum(x_exp, axis=1, keepdims=True)\n",
    "        out = x_exp/(partition+self.epsilon)\n",
    "        \n",
    "        self.mem['out'] = out\n",
    "        self.mem['x_exp'] = x_exp\n",
    "        return out\n",
    "    \n",
    "    def backward(self, grad_y):\n",
    "        '''\n",
    "        grad_y: same shape as x\n",
    "        '''\n",
    "        s = self.mem['out']\n",
    "        sisj = np.matmul(np.expand_dims(s,axis=2), np.expand_dims(s, axis=1)) # (N, c, c)\n",
    "        g_y_exp = np.expand_dims(grad_y, axis=1)\n",
    "        tmp = np.matmul(g_y_exp, sisj) #(N, 1, c)\n",
    "        tmp = np.squeeze(tmp, axis=1)\n",
    "        tmp = -tmp+grad_y*s \n",
    "        return tmp\n",
    "    \n",
    "class Log:\n",
    "    '''\n",
    "    softmax over last dimention\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        self.epsilon = 1e-12\n",
    "        self.mem = {}\n",
    "        \n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        x: shape(N, c)\n",
    "        '''\n",
    "        out = np.log(x+self.epsilon)\n",
    "        \n",
    "        self.mem['x'] = x\n",
    "        return out\n",
    "    \n",
    "    def backward(self, grad_y):\n",
    "        '''\n",
    "        grad_y: same shape as x\n",
    "        '''\n",
    "        x = self.mem['x']\n",
    "        \n",
    "        return 1./(x+1e-12) * grad_y\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[ 2.62322163, -0.87632001, -0.01114105,  1.0353997 ,  2.9326302 ,\n",
      "        -0.43752663],\n",
      "       [ 2.62322163, -0.87632001, -0.01114105,  1.0353997 ,  2.9326302 ,\n",
      "        -0.43752663],\n",
      "       [ 2.62322163, -0.87632001, -0.01114105,  1.0353997 ,  2.9326302 ,\n",
      "        -0.43752663],\n",
      "       [ 2.62322163, -0.87632001, -0.01114105,  1.0353997 ,  2.9326302 ,\n",
      "        -0.43752663],\n",
      "       [ 2.62322163, -0.87632001, -0.01114105,  1.0353997 ,  2.9326302 ,\n",
      "        -0.43752663]]), array([[ 2.23871277,  2.23871277,  2.23871277,  2.23871277],\n",
      "       [ 0.78079526,  0.78079526,  0.78079526,  0.78079526],\n",
      "       [ 0.54071402,  0.54071402,  0.54071402,  0.54071402],\n",
      "       [-1.95633841, -1.95633841, -1.95633841, -1.95633841],\n",
      "       [-0.00834279, -0.00834279, -0.00834279, -0.00834279],\n",
      "       [-3.20235505, -3.20235505, -3.20235505, -3.20235505]]))\n",
      "tf.Tensor(\n",
      "[[ 2.62322163 -0.87632001 -0.01114105  1.0353997   2.9326302  -0.43752663]\n",
      " [ 2.62322163 -0.87632001 -0.01114105  1.0353997   2.9326302  -0.43752663]\n",
      " [ 2.62322163 -0.87632001 -0.01114105  1.0353997   2.9326302  -0.43752663]\n",
      " [ 2.62322163 -0.87632001 -0.01114105  1.0353997   2.9326302  -0.43752663]\n",
      " [ 2.62322163 -0.87632001 -0.01114105  1.0353997   2.9326302  -0.43752663]], shape=(5, 6), dtype=float64)\n",
      "[[1. 1. 0. 0. 1. 0.]\n",
      " [1. 0. 0. 1. 1. 0.]\n",
      " [1. 1. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 1. 1. 0.]\n",
      " [1. 1. 1. 1. 0. 0.]]\n",
      "tf.Tensor(\n",
      "[[1. 1. 0. 0. 1. 0.]\n",
      " [1. 0. 0. 1. 1. 0.]\n",
      " [1. 1. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 1. 1. 0.]\n",
      " [1. 1. 1. 1. 0. 0.]], shape=(5, 6), dtype=float64)\n",
      "[[0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 1.]\n",
      " [1. 0. 0. 0. 0. 0.]]\n",
      "[[-5.29012802e-05  4.84928201e-02 -2.13432066e-04 -4.79334374e-02\n",
      "  -9.50072196e-05 -1.98042169e-04]\n",
      " [ 6.59981286e-04  5.16414585e-06 -1.15042264e-05 -3.54864024e-04\n",
      "  -2.96526324e-04 -2.25085866e-06]\n",
      " [-5.51293696e-09 -3.66187509e-10 -2.99304500e-10  9.71365568e-06\n",
      "  -9.70603165e-06 -1.44560127e-09]\n",
      " [-2.59357614e-08 -2.05998786e-04 -2.10427970e-05 -9.19619889e-11\n",
      "  -2.19798064e-05  2.49047417e-04]\n",
      " [ 3.45214175e-11 -3.32252584e-16 -5.69568548e-17 -3.38821832e-11\n",
      "  -6.32424404e-13 -6.42064179e-15]]\n",
      "tf.Tensor(\n",
      "[[-5.29012802e-05  4.84928201e-02 -2.13432066e-04 -4.79334374e-02\n",
      "  -9.50072196e-05 -1.98042169e-04]\n",
      " [ 6.59981286e-04  5.16414585e-06 -1.15042264e-05 -3.54864024e-04\n",
      "  -2.96526324e-04 -2.25085866e-06]\n",
      " [-5.51293696e-09 -3.66187509e-10 -2.99304500e-10  9.71365568e-06\n",
      "  -9.70603165e-06 -1.44560127e-09]\n",
      " [-2.59357614e-08 -2.05998786e-04 -2.10427970e-05 -9.19619889e-11\n",
      "  -2.19798064e-05  2.49047417e-04]\n",
      " [ 3.45214175e-11 -3.32252584e-16 -5.69568548e-17 -3.38821832e-11\n",
      "  -6.32424404e-13 -6.42064179e-15]], shape=(5, 6), dtype=float64)\n",
      "[[ -0.           4.332462     0.          -0.          -0.\n",
      "   -0.        ]\n",
      " [  1.11726914   2.10491421  -0.           0.          -0.\n",
      "    0.        ]\n",
      " [ -0.           0.          -0.          -8.88716474   0.\n",
      "    0.        ]\n",
      " [ -0.           0.           0.          -0.          -0.\n",
      "  -42.72160215]\n",
      " [ -2.64655984   0.           0.           0.           0.\n",
      "    0.        ]]\n",
      "tf.Tensor(\n",
      "[[ -0.           4.332462     0.          -0.          -0.\n",
      "   -0.        ]\n",
      " [  1.11726914   2.10491421  -0.           0.          -0.\n",
      "    0.        ]\n",
      " [ -0.           0.          -0.          -8.88716474   0.\n",
      "    0.        ]\n",
      " [ -0.           0.           0.          -0.          -0.\n",
      "  -42.72160215]\n",
      " [ -2.64655984   0.           0.           0.           0.\n",
      "    0.        ]], shape=(5, 6), dtype=float64)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\田鸿飞\\AppData\\Local\\Temp\\ipykernel_19132\\360616819.py:92: RuntimeWarning: invalid value encountered in log\n",
      "  out = np.log(x+self.epsilon)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "x = np.random.normal(size=[5, 6])\n",
    "W = np.random.normal(size=[6, 4])\n",
    "aa = Matmul()\n",
    "out = aa.forward(x, W) # shape(5, 4)\n",
    "grad = aa.backward(np.ones_like(out))\n",
    "print (grad)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    x, W = tf.constant(x), tf.constant(W)\n",
    "    tape.watch(x)\n",
    "    y = tf.matmul(x, W)\n",
    "    loss = tf.reduce_sum(y)\n",
    "    grads = tape.gradient(loss, x)\n",
    "    print (grads)\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "x = np.random.normal(size=[5, 6])\n",
    "aa = Relu()\n",
    "out = aa.forward(x) # shape(5, 4)\n",
    "grad = aa.backward(np.ones_like(out))\n",
    "print (grad)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    x= tf.constant(x)\n",
    "    tape.watch(x)\n",
    "    y = tf.nn.relu(x)\n",
    "    loss = tf.reduce_sum(y)\n",
    "    grads = tape.gradient(loss, x)\n",
    "    print (grads)\n",
    "\n",
    "import tensorflow as tf\n",
    "x = np.random.normal(size=[5, 6], scale=5.0, loc=1)\n",
    "label = np.zeros_like(x)\n",
    "label[0, 1]=1.\n",
    "label[1, 0]=1\n",
    "label[1, 1]=1\n",
    "label[2, 3]=1\n",
    "label[3, 5]=1\n",
    "label[4, 0]=1\n",
    "print(label)\n",
    "aa = Softmax()\n",
    "out = aa.forward(x) # shape(5, 6)\n",
    "grad = aa.backward(label)\n",
    "print (grad)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    x= tf.constant(x)\n",
    "    tape.watch(x)\n",
    "    y = tf.nn.softmax(x)\n",
    "    loss = tf.reduce_sum(y*label)\n",
    "    grads = tape.gradient(loss, x)\n",
    "    print (grads)\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "x = np.random.normal(size=[5, 6])\n",
    "aa = Log()\n",
    "out = aa.forward(x) # shape(5, 4)\n",
    "grad = aa.backward(label)\n",
    "print (grad)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    x= tf.constant(x)\n",
    "    tape.watch(x)\n",
    "    y = tf.math.log(x)\n",
    "    loss = tf.reduce_sum(y*label)\n",
    "    grads = tape.gradient(loss, x)\n",
    "    print (grads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Gradient Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00000000e+00 5.33741289e+03 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 0.00000000e+00]\n",
      " [1.23444363e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 0.00000000e+00]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 2.92261896e+01\n",
      "  0.00000000e+00 0.00000000e+00]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 2.60125689e+01]\n",
      " [3.97972899e+02 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 0.00000000e+00]]\n",
      "----------------------------------------\n",
      "[[0.00000000e+00 5.33741291e+03 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 0.00000000e+00]\n",
      " [1.23444363e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 0.00000000e+00]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 2.92261896e+01\n",
      "  0.00000000e+00 0.00000000e+00]\n",
      " [0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 2.60125689e+01]\n",
      " [3.97972900e+02 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      "  0.00000000e+00 0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "label = np.zeros_like(x)\n",
    "label[0, 1]=1.\n",
    "label[1, 0]=1\n",
    "label[2, 3]=1\n",
    "label[3, 5]=1\n",
    "label[4, 0]=1\n",
    "\n",
    "x = np.random.normal(size=[5, 6])\n",
    "W1 = np.random.normal(size=[6, 5])\n",
    "W2 = np.random.normal(size=[5, 6])\n",
    "\n",
    "mul_h1 = Matmul()\n",
    "mul_h2 = Matmul()\n",
    "relu = Relu()\n",
    "softmax = Softmax()\n",
    "log = Log()\n",
    "\n",
    "h1 = mul_h1.forward(x, W1) # shape(5, 4)\n",
    "h1_relu = relu.forward(h1)\n",
    "h2 = mul_h2.forward(h1_relu, W2)\n",
    "h2_soft = softmax.forward(h2)\n",
    "h2_log = log.forward(h2_soft)\n",
    "\n",
    "\n",
    "h2_log_grad = log.backward(label)\n",
    "h2_soft_grad = softmax.backward(h2_log_grad)\n",
    "h2_grad, W2_grad = mul_h2.backward(h2_soft_grad)\n",
    "h1_relu_grad = relu.backward(h2_grad)\n",
    "h1_grad, W1_grad = mul_h1.backward(h1_relu_grad)\n",
    "\n",
    "print(h2_log_grad)\n",
    "print('--'*20)\n",
    "# print(W2_grad)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    x, W1, W2, label = tf.constant(x), tf.constant(W1), tf.constant(W2), tf.constant(label)\n",
    "    tape.watch(W1)\n",
    "    tape.watch(W2)\n",
    "    h1 = tf.matmul(x, W1)\n",
    "    h1_relu = tf.nn.relu(h1)\n",
    "    h2 = tf.matmul(h1_relu, W2)\n",
    "    prob = tf.nn.softmax(h2)\n",
    "    log_prob = tf.math.log(prob)\n",
    "    loss = tf.reduce_sum(label * log_prob)\n",
    "    grads = tape.gradient(loss, [prob])\n",
    "    print (grads[0].numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class myModel:\n",
    "    def __init__(self):\n",
    "        \n",
    "        self.W1 = np.random.normal(size=[28*28+1, 100])\n",
    "        self.W2 = np.random.normal(size=[100, 10])\n",
    "        \n",
    "        self.mul_h1 = Matmul()\n",
    "        self.mul_h2 = Matmul()\n",
    "        self.relu = Relu()\n",
    "        self.softmax = Softmax()\n",
    "        self.log = Log()\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.reshape(-1, 28*28)\n",
    "        bias = np.ones(shape=[x.shape[0], 1])\n",
    "        x = np.concatenate([x, bias], axis=1)\n",
    "        \n",
    "        self.h1 = self.mul_h1.forward(x, self.W1) # shape(5, 4)\n",
    "        self.h1_relu = self.relu.forward(self.h1)\n",
    "        self.h2 = self.mul_h2.forward(self.h1_relu, self.W2)\n",
    "        self.h2_soft = self.softmax.forward(self.h2)\n",
    "        self.h2_log = self.log.forward(self.h2_soft)\n",
    "            \n",
    "    def backward(self, label):\n",
    "        self.h2_log_grad = self.log.backward(-label)\n",
    "        self.h2_soft_grad = self.softmax.backward(self.h2_log_grad)\n",
    "        self.h2_grad, self.W2_grad = self.mul_h2.backward(self.h2_soft_grad)\n",
    "        self.h1_relu_grad = self.relu.backward(self.h2_grad)\n",
    "        self.h1_grad, self.W1_grad = self.mul_h1.backward(self.h1_relu_grad)\n",
    "        \n",
    "model = myModel()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算 loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_loss(log_prob, labels):\n",
    "     return np.mean(np.sum(-log_prob*labels, axis=1))\n",
    "    \n",
    "\n",
    "def compute_accuracy(log_prob, labels):\n",
    "    predictions = np.argmax(log_prob, axis=1)\n",
    "    truth = np.argmax(labels, axis=1)\n",
    "    return np.mean(predictions==truth)\n",
    "\n",
    "def train_one_step(model, x, y):\n",
    "    model.forward(x)\n",
    "    model.backward(y)\n",
    "    model.W1 -= 1e-5* model.W1_grad\n",
    "    model.W2 -= 1e-5* model.W2_grad\n",
    "    loss = compute_loss(model.h2_log, y)\n",
    "    accuracy = compute_accuracy(model.h2_log, y)\n",
    "    return loss, accuracy\n",
    "\n",
    "def test(model, x, y):\n",
    "    model.forward(x)\n",
    "    loss = compute_loss(model.h2_log, y)\n",
    "    accuracy = compute_accuracy(model.h2_log, y)\n",
    "    return loss, accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 实际训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0 : loss 4.0837693873215475 ; accuracy 0.7950666666666667\n",
      "epoch 1 : loss 3.968585589590117 ; accuracy 0.8027333333333333\n",
      "epoch 2 : loss 3.9053463971773885 ; accuracy 0.8026666666666666\n",
      "epoch 3 : loss 3.8770509004945484 ; accuracy 0.8066166666666666\n",
      "epoch 4 : loss 3.791882563602522 ; accuracy 0.8073666666666667\n",
      "epoch 5 : loss 3.752184363453094 ; accuracy 0.8112333333333334\n",
      "epoch 6 : loss 3.658216780548329 ; accuracy 0.8137\n",
      "epoch 7 : loss 3.5876635552590606 ; accuracy 0.8176\n",
      "epoch 8 : loss 3.518727949956535 ; accuracy 0.8198166666666666\n",
      "epoch 9 : loss 3.448087873095096 ; accuracy 0.8237333333333333\n",
      "epoch 10 : loss 3.3868426251373176 ; accuracy 0.8257\n",
      "epoch 11 : loss 3.293536038780456 ; accuracy 0.8299166666666666\n",
      "epoch 12 : loss 3.249697316244106 ; accuracy 0.83195\n",
      "epoch 13 : loss 3.1782779587010275 ; accuracy 0.8349166666666666\n",
      "epoch 14 : loss 3.139000147526382 ; accuracy 0.8365\n",
      "epoch 15 : loss 3.095584508837441 ; accuracy 0.8384666666666667\n",
      "epoch 16 : loss 3.0685860225990984 ; accuracy 0.8398166666666667\n",
      "epoch 17 : loss 3.0344872097771387 ; accuracy 0.8410666666666666\n",
      "epoch 18 : loss 3.0162832467372143 ; accuracy 0.8418333333333333\n",
      "epoch 19 : loss 2.989959953561444 ; accuracy 0.84275\n",
      "epoch 20 : loss 2.9804129852519434 ; accuracy 0.8437666666666667\n",
      "epoch 21 : loss 2.958641486285083 ; accuracy 0.84425\n",
      "epoch 22 : loss 2.9592857997223305 ; accuracy 0.8442833333333334\n",
      "epoch 23 : loss 2.9334732307957285 ; accuracy 0.84495\n",
      "epoch 24 : loss 2.9365533770360037 ; accuracy 0.8452833333333334\n",
      "epoch 25 : loss 2.9028698951117557 ; accuracy 0.8461333333333333\n",
      "epoch 26 : loss 2.900192837717218 ; accuracy 0.8470333333333333\n",
      "epoch 27 : loss 2.8575206286475066 ; accuracy 0.8482833333333333\n",
      "epoch 28 : loss 2.847669557910593 ; accuracy 0.8491\n",
      "epoch 29 : loss 2.7984584742814924 ; accuracy 0.8511166666666666\n",
      "epoch 30 : loss 2.782130532066537 ; accuracy 0.8521\n",
      "epoch 31 : loss 2.7346084744089216 ; accuracy 0.85425\n",
      "epoch 32 : loss 2.716001905714159 ; accuracy 0.8555666666666667\n",
      "epoch 33 : loss 2.6777581197504374 ; accuracy 0.8568333333333333\n",
      "epoch 34 : loss 2.659088724370509 ; accuracy 0.8579333333333333\n",
      "epoch 35 : loss 2.6298952290366353 ; accuracy 0.85925\n",
      "epoch 36 : loss 2.6098321188994573 ; accuracy 0.8602166666666666\n",
      "epoch 37 : loss 2.5871146149875495 ; accuracy 0.8608666666666667\n",
      "epoch 38 : loss 2.5688951996656786 ; accuracy 0.8617333333333334\n",
      "epoch 39 : loss 2.551005901605229 ; accuracy 0.8626\n",
      "epoch 40 : loss 2.5351847525806153 ; accuracy 0.8632\n",
      "epoch 41 : loss 2.5196408650942472 ; accuracy 0.8638833333333333\n",
      "epoch 42 : loss 2.505676331442609 ; accuracy 0.86435\n",
      "epoch 43 : loss 2.4918569557560413 ; accuracy 0.8649333333333333\n",
      "epoch 44 : loss 2.479422902575582 ; accuracy 0.8652666666666666\n",
      "epoch 45 : loss 2.4665498603837697 ; accuracy 0.8660833333333333\n",
      "epoch 46 : loss 2.4554798694239137 ; accuracy 0.86625\n",
      "epoch 47 : loss 2.4428306478720354 ; accuracy 0.8670166666666667\n",
      "epoch 48 : loss 2.432996875292538 ; accuracy 0.8674\n",
      "epoch 49 : loss 2.42098138667365 ; accuracy 0.8679333333333333\n",
      "test loss 2.4050439665774 ; accuracy 0.8709\n"
     ]
    }
   ],
   "source": [
    "train_data, test_data = mnist_dataset()\n",
    "train_label = np.zeros(shape=[train_data[0].shape[0], 10])\n",
    "test_label = np.zeros(shape=[test_data[0].shape[0], 10])\n",
    "train_label[np.arange(train_data[0].shape[0]), np.array(train_data[1])] = 1.\n",
    "test_label[np.arange(test_data[0].shape[0]), np.array(test_data[1])] = 1.\n",
    "\n",
    "for epoch in range(50):\n",
    "    loss, accuracy = train_one_step(model, train_data[0], train_label)\n",
    "    print('epoch', epoch, ': loss', loss, '; accuracy', accuracy)\n",
    "loss, accuracy = test(model, test_data[0], test_label)\n",
    "\n",
    "print('test loss', loss, '; accuracy', accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
